{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import rnn\n",
    "import lstm\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "batch_size = 64\n",
    "theano.config.optimizer='fast_compile'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model = rnn.Rnn(4, 20, 4, lr = 0.01, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n",
    "#model = rnn.RnnMiniBatch(4,20,4, 0.1, single_output=e_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n",
    "#model = lstm.Lstm(4,20,4, 0.1, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)\n",
    "model = lstm.LstmMiniBatch(4, 20, 4, 0.01, batch_size, single_output=False, output_activation='nll', cost_function=T.nnet.softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_short_train = np.load('data/memory_task/short_output_data_test.npy')\n",
    "Y_short_train = np.load('data/memory_task/short_output_data_train.npy')\n",
    "X_short_train = np.load('data/running_max/short_input_data_train.npy')\n",
    "Y_short_train = np.load('data/running_max/short_output_data_train.npy')\n",
    "#Y_short_train = build_preds(X_short_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print X_short_train[0],\"\\n\\n\",Y_short_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_preds(X):\n",
    "    Y = np.zeros(shape=(X.shape[0], X.shape[2]), dtype='float32')\n",
    "    for idx, maxim in enumerate(np.argmax(np.sum(X, axis=1), axis=1)):\n",
    "        Y[idx, maxim] = 1.\n",
    "    return Y\n",
    "\n",
    "def output(X):\n",
    "    Y = np.zeros(shape=X.shape, dtype='float32')\n",
    "    for idx, maxim in enumerate(np.argmax(X, axis=1)):\n",
    "        Y[idx, maxim] = 1.\n",
    "    return Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = 10\n",
    "#minibatch train\n",
    "for epoch in range(epochs):\n",
    "    loss = 0\n",
    "    num_batch = len(X_short_train)/batch_size\n",
    "    for i in range(num_batch):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = min((i+1) * (batch_size), len(X_short_train) -1)\n",
    "        X_train = X_short_train[start_idx:end_idx]\n",
    "        Y_train = Y_short_train[start_idx:end_idx]        \n",
    "        #print X_train.shape, Y_train.shape, i, start_idx\n",
    "        #print model.debug(X_train, Y_train)        \n",
    "        loss += model.train(X_train, Y_train)\n",
    "    print \"epoch \"+str(epoch) + \" sum of batch loss \"+str(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_epochs = 100\n",
    "#naive sgd\n",
    "for x in range(nb_epochs):\n",
    "    loss = 0.\n",
    "    for j in range(len(X_short_train)):  \n",
    "        index = np.random.randint(0, len(X_short_train))\n",
    "        X_train = X_short_train[index]\n",
    "        Y_train = Y_short_train[index]        \n",
    "        loss += model.train(X_train, Y_train)    \n",
    "    print \"epoch \"+str(x)+ \" error: \"+str(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output(model.predictions(X_short_train[0:64]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
